#include "UrlManager.hpp"
#include "util/String.hpp"   // to be config
#include "util/Md5.h"
#include "Host.hpp"

#include <cassert>

using namespace std;

UrlManager::UrlManager(const string& crUrlDbName)
{
    if (!mUrlSeenDb.OpenDb(crUrlDbName)) {
        cerr << "Can not open db!" << endl;
        exit(1);
    }
}

bool
UrlManager::UrlProcess(string& crUrlStr, Host& rHost)
{
    bool status = false;
    if (Normalization(crUrlStr)) {
        if (!IsFiltered(crUrlStr, rHost)) {
            if (!IsUrlVisited(crUrlStr)) {
                status = true;
            } else {
                cout << "\tNOTE: " << crUrlStr << " is already visited!" << endl;
            }
        } else {
            cout << "\tNOTE: " << crUrlStr << " is been filtered by robots.txt!" << endl;
        }
    } else {
        cout << "\tNOTE: " << crUrlStr << " is not well formed!" << endl;
    }
    return status;
}

bool
UrlManager::Normalization(string& rUrl)
{
    bool status = true;
    string::size_type pos_begin, pos_front;
    // to lower case
    String::Str2Lower(rUrl);
    // remove string behind '#'
    pos_begin = rUrl.find('#');
    if (pos_begin != string::npos) {
        rUrl.erase(pos_begin);
    }
    // remove "./" and "../" segment
    while (rUrl.find("/./") != string::npos) {
        pos_begin = rUrl.find("/./");
        if (pos_begin != string::npos) {
            rUrl.erase(pos_begin, 2);
        }
    }

    while (rUrl.find("/../") != string::npos) {
        pos_begin = rUrl.find("/../");
        if (pos_begin != string::npos && pos_begin != 0) {
            pos_front = rUrl.rfind("/", pos_begin - 1);
            if (pos_front != string::npos) {
                rUrl.erase(pos_front, pos_begin - pos_front + 3);
            } else {
                status = false;
                break;
            }
        }
    }
    return status;
}

bool
UrlManager::IsUrlVisited(const string& crUrl)
{
    bool   is_visited = false;
    CMD5   iMD5;
    string digest_str;

    iMD5.GenerateMD5( (unsigned char*)crUrl.c_str(), crUrl.length() );
    digest_str = iMD5.ToString();

    if (mUrlSeenDb.RetrievalData(digest_str).empty()) {
        // not in db
        mUrlSeenDb.AddData(digest_str);
    } else {
        is_visited = true;
    }
    return is_visited;
}

// bool
// UrlManager::ParseUrl(const string& crUrl, Url& rUrlStruct)
// {
//     bool parse_status = false;
//     rUrlStruct.mOrginUrl = crUrl;

//     // only handle http protocol
//     if (crUrl.substr(0, 7) != "http://") {
//         return parse_status;
//     }

//     // parse
//     string::size_type host_start_address;
//     string::size_type host_end_address;
//     string::size_type comma_in_address;

//     host_start_address = crUrl.find("//");
//     assert(host_start_address != string::npos);
//     host_start_address += 2;
//     host_end_address   = crUrl.find("/", host_start_address);
//     comma_in_address   = crUrl.find(":", host_start_address);

//     rUrlStruct.mPort = DEFAULT_HTTP_PORT;

//     if (host_end_address == string::npos) { // eg: http://www.pku.edu.cn(:8080)
//         if (comma_in_address != string::npos) {
//             rUrlStruct.mHost = crUrl.substr(host_start_address,
//                                                  comma_in_address - host_start_address);
//             rUrlStruct.mPort = atoi(crUrl.substr(comma_in_address+1).c_str());
//         } else {
//             rUrlStruct.mHost = crUrl.substr(host_start_address);
//         }
//         rUrlStruct.mPath = "";
//     } else {                    // eg: http://www.pku.edu.cn/(index.html)
//         if (comma_in_address != string::npos) {
//             rUrlStruct.mHost = crUrl.substr(host_start_address,
//                                                  comma_in_address - host_start_address);
//             rUrlStruct.mPort = atoi(crUrl.substr(comma_in_address+1).c_str());
//         } else {
//             rUrlStruct.mHost = crUrl.substr(host_start_address,
//                                                  host_end_address - host_start_address);
//         }
//         rUrlStruct.mPath = crUrl.substr(host_end_address + 1);
//     }

//     if (rUrlStruct.mHost.empty()) {
//         return parse_status;
//     }

//     return true;
// }

bool
UrlManager::IsFiltered(const string& crUrlStruct, Host& crHost)
{
    bool is_filtered = true;

    string::size_type pos;
    const string delims(":/");
    pos = crUrlStruct.find_first_of(delims, 8);
    if (pos == string::npos) {
        return false;
    }

    string host = crUrlStruct.substr(7, pos - 7);
    string path = crUrlStruct.substr(pos);

    vector<string>::iterator it;
    for (it = mGlobalRules.begin(); it != mGlobalRules.end(); ++it) {
        if (path.find(*it) != string::npos) {
            return is_filtered;
        }
    }

    Host_Elem host_elem = crHost.GetElem(host);
    if (host_elem.IP.empty()) {
        return is_filtered;
    } else {
        for (it = host_elem.RobotRules.begin(); it != host_elem.RobotRules.end(); ++it) {
            if (path.find(*it) == 0) {
                return is_filtered;
            }
        }
    }
    return false;
}

